#!/usr/bin/python
# -*- coding: utf-8 -*-

import glob, sys, os, re, string
import pepe.p_txt.bigtxt as bt
import pepe.p_datetime.date_format as df
import pepe.glib as g
import pepe.p_filesystem.dfstats as dfs

import pepe.p_filesystem.basic as pfb

fslist = lambda x, y: pfb.fslist(x, patterns=y, recurse=True, return_folders=False)

files = fslist(  r"""\\server\2010-09-20""" + '\x5C'
              , '*.*')

file_totals = {}
output = open('h01_output.csv', 'wb')
output_dropped = open('h01_dropped.csv', 'wb')
output_totals = open('h01_totals.csv', 'wb')
output_file_totals = open('h01_file_totals.csv', 'wb')
ls = open('h01_list_of_files.csv', 'wb')
output.write(("DT_SOURCE	ACCT_MO	CYCLE	ENTITY	PAGE	SYSTEM	REPORT_ID	TYPE	CURRENCY	RE	EN	AR	SE	CL	REFERENCE_RT	REFERNECE_NUMBER	ISSUE_DATE	ITGN	NAME	CT	XX_ORDER	REFERS_TO	SF	IP	PT	FE	BT	FACE_VALUE	OUTSTANDING_VALUE"+'\x0D\x0A').upper())
       
output_totals.write(("DT_SOURCE" +'\t'+ 'DT_ROW_TYPE'+'\t'+ 'DT_STRUCTURE'+'\t'+ 'DT_SQL_TABLE_NAME'+'\t'+'DT_SOURCE_LINE'
                                         +'\t'+"DT_IS_TOTAL_FOR"+'\t'+"ACCT_MO"+'\t'+"CYCLE"+'\t'+"ENTITY"+'\t'+"PAGE"+'\t'+"SYSTEM"+'\t'+"REPORT_ID"+'\t'
                                         +"TYPE"+'\t'
                                         +"FACE_VALUE"+'\t'
                                         +"OUTSTANDING_VALUE"+'\x0D\x0A').upper())


output_file_totals.write("DT_SOURCE" +'\t'+ "Currency" + '\t'+ "CALCULATED_CURRENCY_TOTAL_FV" + '\t'+ "CURRENCY_SOURCEFILE_TOTAL_FV" + '\t'
                          + "CALCULATED_CURRENCY_TOTAL_OV" + '\t'+ "CURRENCY_SOURCEFILE_TOTAL_OV" + '\x0D\x0A')

DT_STRUCTURE = 'h01'
DT_SQL_TABLE_NAME = ''

for f in files:
    fname = string.split(f, '\x5C')[-1]
    skip_file = 0
    totala = 0
    totalf = 0
    atotals = 0
    invoices_list = []
    invoice_totals = {}

    if 1:        
        fname = string.split(f, '\x5C')[-1]
        #print fname
        if fname[-4:] in ('.tag', 'xlsx', '.xls', ".zip", ".doc", "docx", ".dis", ".mdb", ".xml") or fname[-3:] in ('.gz') or fname[-6:] in ('.accdb'):
            pass
       # elif fname == 'E8MARM1003R.1005.txt':
         #   pass
        
        else:
                process_file = False
                header_type = "RE EN AR SE CL REFERENCE  ISSUE  ITGN    NAME                          CT   XX    REFERS  SF IP PT FE BT    FACE-VALUE  OUTSTANDING"
                file_info = dfs.dfstats(f, options=[])
                bt_lines = bt.lines(f)
                for i in range(12):
                    try:
                        line = re.sub('\x0D\x0A', '\n', bt_lines[i])             

                        if string.count(line, '\x0C') == 0:
                            lines = [line[:]]
                        else:
                            lines = string.split(line, '\x0C')                            
                        for l in lines:
                            if re.search(header_type, l.strip()):
                                process_file = True                               
                    except:
                        continue                    
                if not process_file:
                    pass
                else:                                                   
                    ff = f[f.find('Rawdata'):]
                    file_totals.clear()
                  
                    ACCT_MO = ''
                    CYCLE = '' 
                    ENTITY  = ''
                    PAGE   = ''                
                    SYSTEM  = ''
                    REPORT_ID = ''
                    TYPE = ''
                    CURRENCY = ''
                    RE = ''
                    EN = ''
                    AR= ''
                    SE= ''
                    CL= ''
                    REFERENCE_RT= ''
                    REFERNECE_NUMBER= ''
                    ISSUE_DATE = ''
                    ITGN = ''
                    NAME = ''
                    CT= ''
                    XX_ORDER= '' 
                    REFERS_TO= ''
                    SF = ''
                    IP= ''
                    PT= ''
                    FE = ''
                    BT = ''
                    FACE_VALUE= '' 
                    OUTSTANDING_VALUE= ''

                    ls.write(ff + '\x0D\x0A')
                    print f
                    for line in bt.lines(f):
                        
                        lines = string.split(line, '\x0C')                            
                        for line in lines:
                            
                            line = re.sub('\x0D\x0A', '', line)
                            line = re.sub('\x0C', '', line)
                            line = re.sub('\x0A', '', line)
                            line = re.sub('\x0D', '', line)
                            line = re.sub('\n', '', line)

                            #NEW PAGE HEADER INFO:
                            DT_SOURCE_LINE = line
                            if re.search('^ ACCT-MO', line):
                                ACCT_MO = line[9:18].strip()
                                CYCLE = line[24:30].strip()
                                ENTITY = line[53:76].strip()
                                PAGE = line[127:132].strip()
                               
                                
                            elif re.search('^RUN DATE', line):    
                                SYSTEM = line[41:94].strip()
                                
                            elif re.search('^REPORT-ID', line):    
                                REPORT_ID = line[10:20].strip()
                                TYPE = line[20:100].strip()
                                
                            elif re.search('^                   CURRENCY', line):    
                                CURRENCY = line[28:34].strip()
                            #BODY INFO:                              
                            #elif (re.search('[A-Z0-9]{,2} [A-Z0-9?]{,2} [A-Z0-9]{0,2} \d{6}', line)) or re.search(' {74}CURRENCY',line):                             
                            elif (re.search('[A-Za-z0-9]{,2} [A-Za-z0-9?]{,2} [A-Za-z0-9]{0,2} [A-Za-z0-9]{6}', line) and not re.search(' {65}', line)and not re.search('ENTITY|REFERS|NUMBER', line)) or (re.search(' {74}CURRENCY',line)):
                           
                                RE = line[0:3].strip()
                                EN = line[3:6].strip()
                                AR = line[6:9].strip()
                                SE = line[9:12].strip()
                                CL = line[12:15].strip()
                                REFERENCE_RT = line[15:18].strip()
                                REFERNECE_NUMBER = line[18:26].strip()
                                ISSUE_DATE = line[26:33].strip()
                                ITGN = line[33:41].strip()
                                NAME = line[41:71].strip()
                                CT = line[71:74].strip()
                                XX_ORDER = line[74:82].strip()
                                REFERS_TO = line[82:90].strip()
                                SF = line[90:93].strip()
                                IP = line[93:96].strip()
                                PT = line[96:99].strip()
                                FE = line[99:102].strip()
                                BT = line[102:104].strip()
                                FACE_VALUE = line[104:119].strip()
                                OUTSTANDING_VALUE = line[119:132].strip()
                                IS_TOTAL = False

                                               
                                if RE == EN == AR == SE == CL == REFERENCE_RT == '' and REFERNECE_NUMBER <> '':
                                    IS_TOTAL = 'REFERENCE_NUMBER'
                                if re.search(' {74}CURRENCY', line):
                                    IS_TOTAL = 'CURRENCY_FILE'                                   
                                    
                                if IS_TOTAL in ('REFERENCE_NUMBER','CURRENCY_FILE' ):
                                    if IS_TOTAL in ('REFERENCE_NUMBER'): DT_ROW_TYPE = "TOTAL_PER_REFERENCE_NUMBER"
                                    else: DT_ROW_TYPE = "TOTAL_PER_CURRENCY"
                                        
                                    output_totals.write(ff +'\t'+ DT_ROW_TYPE+'\t'+ DT_STRUCTURE+'\t'+ DT_SQL_TABLE_NAME+'\t'+DT_SOURCE_LINE+'\t'
                                             + IS_TOTAL+'\t'+ACCT_MO+'\t'+CYCLE+'\t'+ENTITY+'\t'+PAGE+'\t'+SYSTEM+'\t'+REPORT_ID+'\t'
                                             +TYPE+'\t'+FACE_VALUE+'\t'+OUTSTANDING_VALUE
                                             +'\x0D\x0A')
                                    
                                    if IS_TOTAL == 'CURRENCY_FILE':
                                        CURRENCY_TOTAL = SF
                                        if file_totals.has_key(CURRENCY_TOTAL):
                                            file_totals[CURRENCY_TOTAL]['CURRENCY_FILE_TOTAL_FV'] = g.string2number(FACE_VALUE)
                                            file_totals[CURRENCY_TOTAL]['CURRENCY_FILE_TOTAL_OV'] = g.string2number(OUTSTANDING_VALUE)

                                        else:
                                            file_totals[CURRENCY_TOTAL] ={}
                                            file_totals[CURRENCY_TOTAL]['CURRENCY_FILE_TOTAL_FV'] = g.string2number(FACE_VALUE)
                                            file_totals[CURRENCY_TOTAL]['CURRENCY_FILE_TOTAL_OV'] = g.string2number(OUTSTANDING_VALUE)

                                        #print  file_totals[CURRENCY_TOTAL]['CURRENCY_FILE_TOTAL_FV'],  file_totals[CURRENCY_TOTAL]['CURRENCY_FILE_TOTAL_OV']     
                                
                                else:
                                    output.write(ff+'\t'+ACCT_MO+'\t'+CYCLE+'\t'+ENTITY+'\t'+PAGE+'\t'+SYSTEM+'\t'+REPORT_ID+'\t'
                                             +TYPE+'\t'+CURRENCY+'\t'+RE+'\t'+EN+'\t'+AR+'\t'+SE+'\t'+CL+'\t'+REFERENCE_RT+'\t'
                                             +REFERNECE_NUMBER+'\t'+ISSUE_DATE+'\t'+ITGN+'\t'+NAME+'\t'+CT+'\t'+XX_ORDER+'\t'
                                             +REFERS_TO+'\t'+SF+'\t'+IP+'\t'+PT+'\t'+FE+'\t'+ BT +'\t'+FACE_VALUE+'\t'
                                             +OUTSTANDING_VALUE+'\x0D\x0A')

                                    if  file_totals.has_key(CURRENCY):
                                        file_totals[CURRENCY]['CURRENCY_TOTAL_FV'] += g.string2number(FACE_VALUE)
                                        file_totals[CURRENCY]['CURRENCY_TOTAL_OV'] += g.string2number(OUTSTANDING_VALUE)
                                                                                     
                                    else:
                                        file_totals[CURRENCY]  = {}
                                        file_totals[CURRENCY]['CURRENCY_TOTAL_FV'] = g.string2number(FACE_VALUE)
                                        file_totals[CURRENCY]['CURRENCY_TOTAL_OV'] = g.string2number(OUTSTANDING_VALUE)
                               
                            else:
                                output_dropped.write(line+'\x0D\x0A')
                                
                    #print file_totals
                    for i in sorted(file_totals.keys()):                       
                        output_file_totals.write(ff + '\t' + i+ '\t' +
                                            str(file_totals[i]['CURRENCY_TOTAL_FV'])+ '\t' +
                                            str(file_totals[i]['CURRENCY_FILE_TOTAL_FV'])+ '\t' +
                                            str(file_totals[i]['CURRENCY_TOTAL_OV'])+ '\t' +
                                            str(file_totals[i]['CURRENCY_FILE_TOTAL_OV'])+ '\x0D\x0A')

output.close()
output_dropped.close()
output_totals.close()
output_file_totals.close()
ls.close()

'''
 ACCT-MO:   0110  CYCLE:   1                  ENTITY:  E8 XX E8 - MCO                        XXX4201P.D.03.03             PAGE:    1
RUN DATE: 11/02/01  17:58:04                       X X X X X X                                                                      
REPORT-ID:11002                             A/R SUB-SYSTEM HEADER SUMMARY REPORT                                                    
              MONTHLY BILLING                                                                                                       
                   CURRENCY:  UD                                                                                                    
                                                                                                                                    
RE EN AR SE CL REFERENCE  ISSUE  ITGN    NAME                          CT   XX    REFERS  SF IP PT FE BT    FACE-VALUE  OUTSTANDING 
               RT NUMBER  DATE                                             ORDER    TO            TX                          VALUE 
------------------------------------------------------------------------------------------------------------------------------------
                                                                                                                                    
   E8 01 HE 1  MI 041001  041001 HE07839 XXXXXX                        06                 53    15    1      -7,618.52     -7,618.52
                  041001                                                                                     -7,618.52     -7,618.52
'''         

                           
